In [1]:
import os
os.chdir('/data/l989o/deployed/a')
In [2]:
import matplotlib.cm
import numpy as np
import time
import torch
from tqdm import tqdm

from torch.utils.data import Dataset, DataLoader
import matplotlib.pyplot as plt
from sklearn.decomposition import PCA

from models.ag_conv_vae_lightning import PerturbedRGBCells
from models.ah_expression_vaes_lightning import PerturbedCellDataset

ds = PerturbedRGBCells(split="validation")

cells_ds = PerturbedCellDataset(split="validation")
if False:
    ds.perturb()
    cells_ds.perturb()

assert torch.all(ds.corrupted_entries == cells_ds.corrupted_entries)

Global seed set to 1234
Global seed set to 1234
In [3]:
models = {
    "resnet_vae": "/data/l989o/deployed/a/data/spatial_uzh_processed/a/checkpoints/resnet_vae/version_7/checkpoints"
    "/epoch=3-step=1610.ckpt",
    "resnet_vae_perturbed": "/data/l989o/deployed/a/data/spatial_uzh_processed/a/checkpoints/resnet_vae/version_12"
    "/checkpoints/last.ckpt",
    "resnet_vae_perturbed_long": "/data/l989o/deployed/a/data/spatial_uzh_processed/a/checkpoints/resnet_vae/version_14/checkpoints/last.ckpt",
    "resnet_vae_last_channel": "/data/l989o/deployed/a/data/spatial_uzh_processed/a/checkpoints/resnet_vae/version_20"
    "/checkpoints/last.ckpt",
}

rgb_ds = ds.rgb_cells
from models.ag_conv_vae_lightning import VAE as ResNetVAE

the_model = "resnet_vae"
# the_model = 'resnet_vae_last_channel'
resnet_vae = ResNetVAE.load_from_checkpoint(models[the_model])
loader = DataLoader(rgb_ds, batch_size=16, num_workers=8, pin_memory=True)
data = loader.__iter__().__next__()
In [4]:
if False:
    start = time.time()
    list_of_z = []
    with torch.no_grad():
        for data in tqdm(loader, desc="embedding the whole validation set"):
            data = [d.to(resnet_vae.device) for d in data]
            z = [zz.cpu() for zz in resnet_vae(*data)]
            list_of_z.append(z)
    print(f"forwarning the data to the resnets: {time.time() - start}")

    torch.cuda.empty_cache()
In [5]:
from data2 import file_path

f = file_path("image_features.npy")
if False:
    mus = torch.cat([zz[2] for zz in list_of_z], dim=0).numpy()
    np.save(f, mus)
mus = np.load(f)
In [6]:
import scanpy as sc
import anndata as ad

a = ad.AnnData(mus)
sc.tl.pca(a)
sc.pl.pca(a)
In [7]:
random_indices = np.random.choice(len(a), 10000, replace=False)
b = a[random_indices]
In [8]:
print("computing umap... ", end="")
sc.pp.neighbors(b)
sc.tl.umap(b)
sc.tl.louvain(b)
print("done")

computing umap... done
In [9]:
plt.figure()
l = b.obs["louvain"].tolist()
colors = list(map(int, l))
plt.scatter(
    b.obsm["X_umap"][:, 0],
    b.obsm["X_umap"][:, 1],
    s=1,
    c=colors,
    cmap=matplotlib.cm.tab20,
)
# plt.xlim([10, 20])
# plt.ylim([0, 10])
plt.show()
In [10]:
from matplotlib.offsetbox import OffsetImage, AnnotationBbox

fig, ax = plt.subplots(figsize=(24, 14))
u = b.obsm["X_umap"]
for j, i in enumerate(tqdm(random_indices[:500], desc="scatterplot with images")):
    ome, mask, _ = ds[i]
    ome = ome.numpy()
    mask = torch.squeeze(mask, 0).numpy()
    im = OffsetImage(mask, zoom=0.7)
    ab = AnnotationBbox(im, u[j], xycoords="data", frameon=False)
    ax.add_artist(ab)
ax.set(xlim=(min(u[:, 0]), max(u[:, 0])), ylim=(min(u[:, 1]), max(u[:, 1])))
plt.tight_layout()
plt.show()

scatterplot with images: 100%|██████████| 500/500 [00:01<00:00, 340.73it/s]
In [11]:
p = 5000
# n_channels = ds[42][0].shape[0]
# channels = list(range(n_channels))
channels = [10, 34, 35, 38]
#     ax.autoscale()
for c in tqdm(channels, desc='channels'):
    fig, ax = plt.subplots(figsize=(24, 14))
    ax.set(title=f'ch {c}')
    u = b.obsm["X_umap"]
    for j, i in enumerate(tqdm(random_indices[:p], desc="scatterplot with images")):
        ome, _, _ = ds[i]
        ome = ome[c, :, :].numpy()
        # mask = torch.squeeze(mask, 0).numpy()
        im = OffsetImage(ome, zoom=0.7)
        ab = AnnotationBbox(im, u[j], xycoords="data", frameon=False)
        ax.add_artist(ab)
    ax.set(xlim=(min(u[:, 0]), max(u[:, 0])), ylim=(min(u[:, 1]), max(u[:, 1])))
    plt.tight_layout()
    plt.show()

channels:   0%|          | 0/4 [00:00<?, ?it/s]
scatterplot with images:   0%|          | 0/5000 [00:00<?, ?it/s]
scatterplot with images:   1%|          | 34/5000 [00:00<00:14, 331.71it/s]
scatterplot with images:   1%|▏         | 69/5000 [00:00<00:14, 334.99it/s]
scatterplot with images:   2%|▏         | 104/5000 [00:00<00:14, 337.62it/s]
scatterplot with images:   3%|▎         | 138/5000 [00:00<00:14, 337.87it/s]
scatterplot with images:   3%|▎         | 172/5000 [00:00<00:14, 337.12it/s]
scatterplot with images:   4%|▍         | 206/5000 [00:00<00:14, 337.42it/s]
scatterplot with images:   5%|▍         | 240/5000 [00:00<00:14, 336.70it/s]
scatterplot with images:   5%|▌         | 274/5000 [00:00<00:14, 337.27it/s]
scatterplot with images:   6%|▌         | 308/5000 [00:00<00:13, 337.48it/s]
scatterplot with images:   7%|▋         | 341/5000 [00:01<00:14, 328.61it/s]
scatterplot with images:   8%|▊         | 375/5000 [00:01<00:13, 330.78it/s]
scatterplot with images:   8%|▊         | 409/5000 [00:01<00:13, 331.79it/s]
scatterplot with images:   9%|▉         | 443/5000 [00:01<00:13, 333.66it/s]
scatterplot with images:  10%|▉         | 485/5000 [00:01<00:12, 352.96it/s]
scatterplot with images:  11%|█         | 528/5000 [00:01<00:12, 372.32it/s]
scatterplot with images:  11%|█▏        | 571/5000 [00:01<00:11, 386.52it/s]
scatterplot with images:  12%|█▏        | 613/5000 [00:01<00:11, 395.79it/s]
scatterplot with images:  13%|█▎        | 656/5000 [00:01<00:10, 404.46it/s]
scatterplot with images:  14%|█▍        | 699/5000 [00:01<00:10, 409.29it/s]
scatterplot with images:  15%|█▍        | 741/5000 [00:02<00:10, 406.96it/s]
scatterplot with images:  16%|█▌        | 784/5000 [00:02<00:10, 411.39it/s]
scatterplot with images:  17%|█▋        | 827/5000 [00:02<00:10, 414.67it/s]
scatterplot with images:  17%|█▋        | 870/5000 [00:02<00:09, 416.72it/s]
scatterplot with images:  18%|█▊        | 913/5000 [00:02<00:09, 418.84it/s]
scatterplot with images:  19%|█▉        | 956/5000 [00:02<00:09, 419.62it/s]
scatterplot with images:  20%|█▉        | 999/5000 [00:02<00:09, 420.85it/s]
scatterplot with images:  21%|██        | 1042/5000 [00:02<00:09, 420.05it/s]
scatterplot with images:  22%|██▏       | 1085/5000 [00:02<00:09, 420.40it/s]
scatterplot with images:  23%|██▎       | 1128/5000 [00:02<00:09, 418.87it/s]
scatterplot with images:  23%|██▎       | 1170/5000 [00:03<00:09, 412.55it/s]
scatterplot with images:  24%|██▍       | 1212/5000 [00:03<00:09, 414.63it/s]
scatterplot with images:  25%|██▌       | 1255/5000 [00:03<00:08, 416.31it/s]
scatterplot with images:  26%|██▌       | 1297/5000 [00:03<00:08, 417.06it/s]
scatterplot with images:  27%|██▋       | 1340/5000 [00:03<00:08, 418.13it/s]
scatterplot with images:  28%|██▊       | 1382/5000 [00:03<00:14, 251.29it/s]
scatterplot with images:  28%|██▊       | 1424/5000 [00:03<00:12, 285.68it/s]
scatterplot with images:  29%|██▉       | 1466/5000 [00:03<00:11, 314.75it/s]
scatterplot with images:  30%|███       | 1506/5000 [00:04<00:10, 334.44it/s]
scatterplot with images:  31%|███       | 1548/5000 [00:04<00:09, 354.85it/s]
scatterplot with images:  32%|███▏      | 1590/5000 [00:04<00:09, 371.82it/s]
scatterplot with images:  33%|███▎      | 1633/5000 [00:04<00:08, 385.21it/s]
scatterplot with images:  34%|███▎      | 1676/5000 [00:04<00:08, 396.42it/s]
scatterplot with images:  34%|███▍      | 1719/5000 [00:04<00:08, 403.93it/s]
scatterplot with images:  35%|███▌      | 1762/5000 [00:04<00:07, 409.60it/s]
scatterplot with images:  36%|███▌      | 1805/5000 [00:04<00:07, 413.98it/s]
scatterplot with images:  37%|███▋      | 1848/5000 [00:04<00:07, 416.31it/s]
scatterplot with images:  38%|███▊      | 1891/5000 [00:05<00:07, 417.25it/s]
scatterplot with images:  39%|███▊      | 1933/5000 [00:05<00:07, 410.51it/s]
scatterplot with images:  40%|███▉      | 1975/5000 [00:05<00:07, 413.10it/s]
scatterplot with images:  40%|████      | 2018/5000 [00:05<00:07, 415.26it/s]
scatterplot with images:  41%|████      | 2061/5000 [00:05<00:07, 416.77it/s]
scatterplot with images:  42%|████▏     | 2103/5000 [00:05<00:06, 416.67it/s]
scatterplot with images:  43%|████▎     | 2145/5000 [00:05<00:06, 417.52it/s]
scatterplot with images:  44%|████▎     | 2187/5000 [00:05<00:06, 417.40it/s]
scatterplot with images:  45%|████▍     | 2229/5000 [00:05<00:06, 417.49it/s]
scatterplot with images:  45%|████▌     | 2271/5000 [00:05<00:06, 418.22it/s]
scatterplot with images:  46%|████▋     | 2313/5000 [00:06<00:06, 409.74it/s]
scatterplot with images:  47%|████▋     | 2355/5000 [00:06<00:06, 411.88it/s]
scatterplot with images:  48%|████▊     | 2397/5000 [00:06<00:06, 413.23it/s]
scatterplot with images:  49%|████▉     | 2439/5000 [00:06<00:06, 415.01it/s]
scatterplot with images:  50%|████▉     | 2481/5000 [00:06<00:06, 415.36it/s]
scatterplot with images:  50%|█████     | 2523/5000 [00:06<00:05, 416.54it/s]
scatterplot with images:  51%|█████▏    | 2565/5000 [00:06<00:05, 417.00it/s]
scatterplot with images:  52%|█████▏    | 2607/5000 [00:06<00:05, 417.67it/s]
scatterplot with images:  53%|█████▎    | 2649/5000 [00:06<00:05, 416.63it/s]
scatterplot with images:  54%|█████▍    | 2691/5000 [00:06<00:05, 417.27it/s]
scatterplot with images:  55%|█████▍    | 2733/5000 [00:07<00:05, 409.52it/s]
scatterplot with images:  56%|█████▌    | 2775/5000 [00:07<00:05, 411.94it/s]
scatterplot with images:  56%|█████▋    | 2817/5000 [00:07<00:05, 412.86it/s]
scatterplot with images:  57%|█████▋    | 2859/5000 [00:07<00:05, 414.50it/s]
scatterplot with images:  58%|█████▊    | 2901/5000 [00:07<00:05, 415.14it/s]
scatterplot with images:  59%|█████▉    | 2944/5000 [00:07<00:04, 416.65it/s]
scatterplot with images:  60%|█████▉    | 2986/5000 [00:07<00:04, 416.10it/s]
scatterplot with images:  61%|██████    | 3028/5000 [00:07<00:04, 417.07it/s]
scatterplot with images:  61%|██████▏   | 3070/5000 [00:07<00:04, 416.67it/s]
scatterplot with images:  62%|██████▏   | 3112/5000 [00:07<00:04, 417.57it/s]
scatterplot with images:  63%|██████▎   | 3154/5000 [00:08<00:04, 409.39it/s]
scatterplot with images:  64%|██████▍   | 3196/5000 [00:08<00:04, 411.76it/s]
scatterplot with images:  65%|██████▍   | 3238/5000 [00:08<00:04, 413.08it/s]
scatterplot with images:  66%|██████▌   | 3280/5000 [00:08<00:04, 414.88it/s]
scatterplot with images:  66%|██████▋   | 3322/5000 [00:08<00:04, 415.22it/s]
scatterplot with images:  67%|██████▋   | 3364/5000 [00:08<00:03, 416.05it/s]
scatterplot with images:  68%|██████▊   | 3406/5000 [00:08<00:03, 416.12it/s]
scatterplot with images:  69%|██████▉   | 3448/5000 [00:09<00:06, 241.58it/s]
scatterplot with images:  70%|██████▉   | 3487/5000 [00:09<00:05, 272.65it/s]
scatterplot with images:  71%|███████   | 3529/5000 [00:09<00:04, 303.83it/s]
scatterplot with images:  71%|███████▏  | 3571/5000 [00:09<00:04, 330.68it/s]
scatterplot with images:  72%|███████▏  | 3613/5000 [00:09<00:03, 352.72it/s]
scatterplot with images:  73%|███████▎  | 3655/5000 [00:09<00:03, 368.44it/s]
scatterplot with images:  74%|███████▍  | 3697/5000 [00:09<00:03, 381.59it/s]
scatterplot with images:  75%|███████▍  | 3739/5000 [00:09<00:03, 390.66it/s]
scatterplot with images:  76%|███████▌  | 3781/5000 [00:09<00:03, 398.67it/s]
scatterplot with images:  76%|███████▋  | 3823/5000 [00:09<00:02, 402.81it/s]
scatterplot with images:  77%|███████▋  | 3865/5000 [00:10<00:02, 398.43it/s]
scatterplot with images:  78%|███████▊  | 3907/5000 [00:10<00:02, 403.68it/s]
scatterplot with images:  79%|███████▉  | 3949/5000 [00:10<00:02, 406.34it/s]
scatterplot with images:  80%|███████▉  | 3991/5000 [00:10<00:02, 408.74it/s]
scatterplot with images:  81%|████████  | 4033/5000 [00:10<00:02, 410.28it/s]
scatterplot with images:  82%|████████▏ | 4075/5000 [00:10<00:02, 410.21it/s]
scatterplot with images:  82%|████████▏ | 4117/5000 [00:10<00:02, 411.29it/s]
scatterplot with images:  83%|████████▎ | 4159/5000 [00:10<00:02, 411.69it/s]
scatterplot with images:  84%|████████▍ | 4201/5000 [00:10<00:01, 412.89it/s]
scatterplot with images:  85%|████████▍ | 4243/5000 [00:10<00:01, 411.62it/s]
scatterplot with images:  86%|████████▌ | 4285/5000 [00:11<00:01, 404.57it/s]
scatterplot with images:  87%|████████▋ | 4327/5000 [00:11<00:01, 407.00it/s]
scatterplot with images:  87%|████████▋ | 4369/5000 [00:11<00:01, 408.35it/s]
scatterplot with images:  88%|████████▊ | 4410/5000 [00:11<00:01, 408.62it/s]
scatterplot with images:  89%|████████▉ | 4452/5000 [00:11<00:01, 410.59it/s]
scatterplot with images:  90%|████████▉ | 4494/5000 [00:11<00:01, 411.00it/s]
scatterplot with images:  91%|█████████ | 4536/5000 [00:11<00:01, 411.92it/s]
scatterplot with images:  92%|█████████▏| 4578/5000 [00:11<00:01, 412.23it/s]
scatterplot with images:  92%|█████████▏| 4620/5000 [00:11<00:00, 411.64it/s]
scatterplot with images:  93%|█████████▎| 4662/5000 [00:11<00:00, 412.77it/s]
scatterplot with images:  94%|█████████▍| 4704/5000 [00:12<00:00, 405.27it/s]
scatterplot with images:  95%|█████████▍| 4745/5000 [00:12<00:00, 406.09it/s]
scatterplot with images:  96%|█████████▌| 4787/5000 [00:12<00:00, 407.26it/s]
scatterplot with images:  97%|█████████▋| 4829/5000 [00:12<00:00, 409.60it/s]
scatterplot with images:  97%|█████████▋| 4871/5000 [00:12<00:00, 410.72it/s]
scatterplot with images:  98%|█████████▊| 4913/5000 [00:12<00:00, 410.87it/s]
scatterplot with images:  99%|█████████▉| 4955/5000 [00:12<00:00, 410.63it/s]
scatterplot with images: 100%|██████████| 5000/5000 [00:12<00:00, 391.13it/s]

channels:  25%|██▌       | 1/4 [00:25<01:17, 25.87s/it]
scatterplot with images:   0%|          | 0/5000 [00:00<?, ?it/s]
scatterplot with images:   1%|          | 39/5000 [00:00<00:12, 386.50it/s]
scatterplot with images:   2%|▏         | 82/5000 [00:00<00:12, 396.06it/s]
scatterplot with images:   2%|▏         | 124/5000 [00:00<00:12, 402.49it/s]
scatterplot with images:   3%|▎         | 165/5000 [00:00<00:11, 404.23it/s]
scatterplot with images:   4%|▍         | 207/5000 [00:00<00:11, 407.42it/s]
scatterplot with images:   5%|▍         | 248/5000 [00:00<00:11, 407.81it/s]
scatterplot with images:   6%|▌         | 290/5000 [00:00<00:11, 409.84it/s]
scatterplot with images:   7%|▋         | 331/5000 [00:00<00:11, 409.65it/s]
scatterplot with images:   7%|▋         | 373/5000 [00:00<00:11, 410.21it/s]
scatterplot with images:   8%|▊         | 413/5000 [00:01<00:11, 402.17it/s]
scatterplot with images:   9%|▉         | 455/5000 [00:01<00:11, 405.37it/s]
scatterplot with images:  10%|▉         | 496/5000 [00:01<00:11, 406.74it/s]
scatterplot with images:  11%|█         | 538/5000 [00:01<00:10, 408.81it/s]
scatterplot with images:  12%|█▏        | 579/5000 [00:01<00:10, 407.96it/s]
scatterplot with images:  12%|█▏        | 621/5000 [00:01<00:10, 409.43it/s]
scatterplot with images:  13%|█▎        | 663/5000 [00:01<00:10, 410.08it/s]
scatterplot with images:  14%|█▍        | 705/5000 [00:01<00:10, 410.80it/s]
scatterplot with images:  15%|█▍        | 747/5000 [00:01<00:10, 411.08it/s]
scatterplot with images:  16%|█▌        | 789/5000 [00:01<00:10, 411.01it/s]
scatterplot with images:  17%|█▋        | 831/5000 [00:02<00:10, 403.73it/s]
scatterplot with images:  17%|█▋        | 873/5000 [00:02<00:10, 406.39it/s]
scatterplot with images:  18%|█▊        | 915/5000 [00:02<00:10, 407.66it/s]
scatterplot with images:  19%|█▉        | 957/5000 [00:02<00:09, 409.69it/s]
scatterplot with images:  20%|█▉        | 998/5000 [00:02<00:09, 409.35it/s]
scatterplot with images:  21%|██        | 1040/5000 [00:02<00:09, 410.22it/s]
scatterplot with images:  22%|██▏       | 1082/5000 [00:02<00:09, 409.25it/s]
scatterplot with images:  22%|██▏       | 1123/5000 [00:02<00:09, 408.47it/s]
scatterplot with images:  23%|██▎       | 1165/5000 [00:02<00:09, 410.13it/s]
scatterplot with images:  24%|██▍       | 1207/5000 [00:02<00:09, 408.86it/s]
scatterplot with images:  25%|██▍       | 1248/5000 [00:03<00:09, 402.85it/s]
scatterplot with images:  26%|██▌       | 1289/5000 [00:03<00:09, 404.04it/s]
scatterplot with images:  27%|██▋       | 1331/5000 [00:03<00:09, 405.85it/s]
scatterplot with images:  27%|██▋       | 1372/5000 [00:03<00:08, 406.68it/s]
scatterplot with images:  28%|██▊       | 1413/5000 [00:03<00:08, 407.64it/s]
scatterplot with images:  29%|██▉       | 1454/5000 [00:03<00:08, 407.43it/s]
scatterplot with images:  30%|██▉       | 1495/5000 [00:03<00:08, 407.68it/s]
scatterplot with images:  31%|███       | 1536/5000 [00:03<00:08, 407.29it/s]
scatterplot with images:  32%|███▏      | 1577/5000 [00:03<00:08, 407.59it/s]
scatterplot with images:  32%|███▏      | 1618/5000 [00:03<00:08, 407.01it/s]
scatterplot with images:  33%|███▎      | 1659/5000 [00:04<00:08, 400.49it/s]
scatterplot with images:  34%|███▍      | 1700/5000 [00:04<00:08, 402.67it/s]
scatterplot with images:  35%|███▍      | 1741/5000 [00:04<00:08, 404.05it/s]
scatterplot with images:  36%|███▌      | 1783/5000 [00:04<00:07, 406.05it/s]
scatterplot with images:  36%|███▋      | 1825/5000 [00:04<00:07, 407.32it/s]
scatterplot with images:  37%|███▋      | 1867/5000 [00:04<00:07, 409.18it/s]
scatterplot with images:  38%|███▊      | 1908/5000 [00:05<00:19, 158.99it/s]
scatterplot with images:  39%|███▉      | 1946/5000 [00:05<00:15, 192.52it/s]
scatterplot with images:  40%|███▉      | 1985/5000 [00:05<00:13, 226.07it/s]
scatterplot with images:  40%|████      | 2025/5000 [00:05<00:11, 259.09it/s]
scatterplot with images:  41%|████▏     | 2064/5000 [00:05<00:10, 287.23it/s]
scatterplot with images:  42%|████▏     | 2104/5000 [00:05<00:09, 312.03it/s]
scatterplot with images:  43%|████▎     | 2143/5000 [00:05<00:08, 331.54it/s]
scatterplot with images:  44%|████▎     | 2182/5000 [00:05<00:08, 345.97it/s]
scatterplot with images:  44%|████▍     | 2221/5000 [00:06<00:07, 356.88it/s]
scatterplot with images:  45%|████▌     | 2261/5000 [00:06<00:07, 361.61it/s]
scatterplot with images:  46%|████▌     | 2301/5000 [00:06<00:07, 370.15it/s]
scatterplot with images:  47%|████▋     | 2341/5000 [00:06<00:07, 377.80it/s]
scatterplot with images:  48%|████▊     | 2381/5000 [00:06<00:06, 382.97it/s]
scatterplot with images:  48%|████▊     | 2421/5000 [00:06<00:06, 387.77it/s]
scatterplot with images:  49%|████▉     | 2461/5000 [00:06<00:06, 390.86it/s]
scatterplot with images:  50%|█████     | 2502/5000 [00:06<00:06, 394.57it/s]
scatterplot with images:  51%|█████     | 2543/5000 [00:06<00:06, 397.09it/s]
scatterplot with images:  52%|█████▏    | 2584/5000 [00:06<00:06, 399.29it/s]
scatterplot with images:  52%|█████▎    | 2625/5000 [00:07<00:05, 400.45it/s]
scatterplot with images:  53%|█████▎    | 2666/5000 [00:07<00:05, 394.52it/s]
scatterplot with images:  54%|█████▍    | 2707/5000 [00:07<00:05, 397.25it/s]
scatterplot with images:  55%|█████▍    | 2748/5000 [00:07<00:05, 399.96it/s]
scatterplot with images:  56%|█████▌    | 2789/5000 [00:07<00:05, 402.10it/s]
scatterplot with images:  57%|█████▋    | 2830/5000 [00:07<00:05, 402.95it/s]
scatterplot with images:  57%|█████▋    | 2871/5000 [00:07<00:05, 402.99it/s]
scatterplot with images:  58%|█████▊    | 2912/5000 [00:07<00:05, 402.45it/s]
scatterplot with images:  59%|█████▉    | 2953/5000 [00:07<00:05, 402.86it/s]
scatterplot with images:  60%|█████▉    | 2994/5000 [00:07<00:04, 402.74it/s]
scatterplot with images:  61%|██████    | 3035/5000 [00:08<00:04, 404.29it/s]
scatterplot with images:  62%|██████▏   | 3076/5000 [00:08<00:04, 397.69it/s]
scatterplot with images:  62%|██████▏   | 3118/5000 [00:08<00:04, 401.50it/s]
scatterplot with images:  63%|██████▎   | 3159/5000 [00:08<00:04, 403.97it/s]
scatterplot with images:  64%|██████▍   | 3201/5000 [00:08<00:04, 406.01it/s]
scatterplot with images:  65%|██████▍   | 3242/5000 [00:08<00:04, 407.14it/s]
scatterplot with images:  66%|██████▌   | 3284/5000 [00:08<00:04, 409.11it/s]
scatterplot with images:  66%|██████▋   | 3325/5000 [00:08<00:04, 408.87it/s]
scatterplot with images:  67%|██████▋   | 3366/5000 [00:08<00:04, 408.27it/s]
scatterplot with images:  68%|██████▊   | 3408/5000 [00:08<00:03, 409.25it/s]
scatterplot with images:  69%|██████▉   | 3450/5000 [00:09<00:03, 409.94it/s]
scatterplot with images:  70%|██████▉   | 3491/5000 [00:09<00:03, 403.41it/s]
scatterplot with images:  71%|███████   | 3533/5000 [00:09<00:03, 406.76it/s]
scatterplot with images:  72%|███████▏  | 3575/5000 [00:09<00:03, 408.00it/s]
scatterplot with images:  72%|███████▏  | 3617/5000 [00:09<00:03, 409.87it/s]
scatterplot with images:  73%|███████▎  | 3659/5000 [00:09<00:03, 411.74it/s]
scatterplot with images:  74%|███████▍  | 3701/5000 [00:09<00:03, 412.49it/s]
scatterplot with images:  75%|███████▍  | 3743/5000 [00:09<00:03, 413.62it/s]
scatterplot with images:  76%|███████▌  | 3785/5000 [00:09<00:02, 414.63it/s]
scatterplot with images:  77%|███████▋  | 3828/5000 [00:09<00:02, 416.92it/s]
scatterplot with images:  77%|███████▋  | 3870/5000 [00:10<00:02, 415.11it/s]
scatterplot with images:  78%|███████▊  | 3912/5000 [00:10<00:02, 409.38it/s]
scatterplot with images:  79%|███████▉  | 3954/5000 [00:10<00:02, 411.25it/s]
scatterplot with images:  80%|███████▉  | 3997/5000 [00:10<00:02, 414.05it/s]
scatterplot with images:  81%|████████  | 4039/5000 [00:10<00:04, 234.66it/s]
scatterplot with images:  82%|████████▏ | 4081/5000 [00:10<00:03, 270.38it/s]
scatterplot with images:  82%|████████▏ | 4123/5000 [00:10<00:02, 302.49it/s]
scatterplot with images:  83%|████████▎ | 4165/5000 [00:11<00:02, 330.20it/s]
scatterplot with images:  84%|████████▍ | 4205/5000 [00:11<00:02, 346.87it/s]
scatterplot with images:  85%|████████▍ | 4248/5000 [00:11<00:02, 366.38it/s]
scatterplot with images:  86%|████████▌ | 4291/5000 [00:11<00:01, 381.00it/s]
scatterplot with images:  87%|████████▋ | 4334/5000 [00:11<00:01, 392.32it/s]
scatterplot with images:  88%|████████▊ | 4377/5000 [00:11<00:01, 401.46it/s]
scatterplot with images:  88%|████████▊ | 4419/5000 [00:11<00:01, 406.79it/s]
scatterplot with images:  89%|████████▉ | 4462/5000 [00:11<00:01, 411.59it/s]
scatterplot with images:  90%|█████████ | 4505/5000 [00:11<00:01, 414.70it/s]
scatterplot with images:  91%|█████████ | 4548/5000 [00:11<00:01, 418.08it/s]
scatterplot with images:  92%|█████████▏| 4591/5000 [00:12<00:00, 418.00it/s]
scatterplot with images:  93%|█████████▎| 4634/5000 [00:12<00:00, 411.06it/s]
scatterplot with images:  94%|█████████▎| 4676/5000 [00:12<00:00, 413.19it/s]
scatterplot with images:  94%|█████████▍| 4719/5000 [00:12<00:00, 415.21it/s]
scatterplot with images:  95%|█████████▌| 4762/5000 [00:12<00:00, 416.95it/s]
scatterplot with images:  96%|█████████▌| 4805/5000 [00:12<00:00, 418.81it/s]
scatterplot with images:  97%|█████████▋| 4847/5000 [00:12<00:00, 419.05it/s]
scatterplot with images:  98%|█████████▊| 4890/5000 [00:12<00:00, 419.39it/s]
scatterplot with images:  99%|█████████▊| 4932/5000 [00:12<00:00, 419.18it/s]
scatterplot with images: 100%|██████████| 5000/5000 [00:13<00:00, 382.98it/s]

channels:  50%|█████     | 2/4 [00:51<00:51, 25.81s/it]
scatterplot with images:   0%|          | 0/5000 [00:00<?, ?it/s]
scatterplot with images:   1%|          | 39/5000 [00:00<00:12, 383.26it/s]
scatterplot with images:   2%|▏         | 81/5000 [00:00<00:12, 392.56it/s]
scatterplot with images:   2%|▏         | 124/5000 [00:00<00:12, 401.28it/s]
scatterplot with images:   3%|▎         | 166/5000 [00:00<00:11, 405.97it/s]
scatterplot with images:   4%|▍         | 209/5000 [00:00<00:11, 411.97it/s]
scatterplot with images:   5%|▌         | 251/5000 [00:00<00:11, 412.32it/s]
scatterplot with images:   6%|▌         | 293/5000 [00:00<00:11, 414.21it/s]
scatterplot with images:   7%|▋         | 335/5000 [00:00<00:11, 415.07it/s]
scatterplot with images:   8%|▊         | 378/5000 [00:00<00:11, 416.56it/s]
scatterplot with images:   8%|▊         | 419/5000 [00:01<00:11, 410.16it/s]
scatterplot with images:   9%|▉         | 462/5000 [00:01<00:10, 413.15it/s]
scatterplot with images:  10%|█         | 504/5000 [00:01<00:10, 413.98it/s]
scatterplot with images:  11%|█         | 547/5000 [00:01<00:10, 416.89it/s]
scatterplot with images:  12%|█▏        | 589/5000 [00:01<00:10, 417.57it/s]
scatterplot with images:  13%|█▎        | 632/5000 [00:01<00:10, 418.76it/s]
scatterplot with images:  14%|█▎        | 675/5000 [00:01<00:10, 419.66it/s]
scatterplot with images:  14%|█▍        | 718/5000 [00:01<00:10, 420.30it/s]
scatterplot with images:  15%|█▌        | 760/5000 [00:01<00:10, 419.94it/s]
scatterplot with images:  16%|█▌        | 802/5000 [00:01<00:10, 418.78it/s]
scatterplot with images:  17%|█▋        | 844/5000 [00:02<00:10, 412.13it/s]
scatterplot with images:  18%|█▊        | 887/5000 [00:02<00:09, 415.18it/s]
scatterplot with images:  19%|█▊        | 930/5000 [00:02<00:09, 417.48it/s]
scatterplot with images:  19%|█▉        | 972/5000 [00:02<00:09, 417.89it/s]
scatterplot with images:  20%|██        | 1014/5000 [00:02<00:09, 416.35it/s]
scatterplot with images:  21%|██        | 1056/5000 [00:02<00:09, 415.78it/s]
scatterplot with images:  22%|██▏       | 1099/5000 [00:02<00:09, 417.28it/s]
scatterplot with images:  23%|██▎       | 1141/5000 [00:02<00:09, 417.19it/s]
scatterplot with images:  24%|██▎       | 1183/5000 [00:02<00:09, 415.51it/s]
scatterplot with images:  24%|██▍       | 1225/5000 [00:02<00:09, 414.32it/s]
scatterplot with images:  25%|██▌       | 1267/5000 [00:03<00:09, 402.30it/s]
scatterplot with images:  26%|██▌       | 1309/5000 [00:03<00:09, 406.25it/s]
scatterplot with images:  27%|██▋       | 1351/5000 [00:03<00:08, 408.44it/s]
scatterplot with images:  28%|██▊       | 1393/5000 [00:03<00:08, 409.62it/s]
scatterplot with images:  29%|██▊       | 1435/5000 [00:03<00:08, 412.43it/s]
scatterplot with images:  30%|██▉       | 1477/5000 [00:03<00:08, 411.29it/s]
scatterplot with images:  30%|███       | 1519/5000 [00:03<00:08, 410.10it/s]
scatterplot with images:  31%|███       | 1561/5000 [00:03<00:08, 410.35it/s]
scatterplot with images:  32%|███▏      | 1603/5000 [00:03<00:08, 411.61it/s]
scatterplot with images:  33%|███▎      | 1646/5000 [00:03<00:08, 415.03it/s]
scatterplot with images:  34%|███▍      | 1688/5000 [00:04<00:08, 404.94it/s]
scatterplot with images:  35%|███▍      | 1730/5000 [00:04<00:08, 407.89it/s]
scatterplot with images:  35%|███▌      | 1772/5000 [00:04<00:07, 410.74it/s]
scatterplot with images:  36%|███▋      | 1814/5000 [00:04<00:07, 411.01it/s]
scatterplot with images:  37%|███▋      | 1856/5000 [00:04<00:07, 412.61it/s]
scatterplot with images:  38%|███▊      | 1898/5000 [00:04<00:07, 409.96it/s]
scatterplot with images:  39%|███▉      | 1940/5000 [00:04<00:07, 408.55it/s]
scatterplot with images:  40%|███▉      | 1981/5000 [00:04<00:07, 407.08it/s]
scatterplot with images:  40%|████      | 2022/5000 [00:04<00:07, 404.51it/s]
scatterplot with images:  41%|████▏     | 2063/5000 [00:05<00:07, 404.13it/s]
scatterplot with images:  42%|████▏     | 2104/5000 [00:05<00:07, 397.75it/s]
scatterplot with images:  43%|████▎     | 2144/5000 [00:05<00:07, 398.30it/s]
scatterplot with images:  44%|████▎     | 2185/5000 [00:05<00:07, 400.63it/s]
scatterplot with images:  45%|████▍     | 2226/5000 [00:05<00:06, 400.38it/s]
scatterplot with images:  45%|████▌     | 2267/5000 [00:05<00:06, 401.41it/s]
scatterplot with images:  46%|████▌     | 2308/5000 [00:05<00:06, 399.64it/s]
scatterplot with images:  47%|████▋     | 2349/5000 [00:05<00:06, 402.31it/s]
scatterplot with images:  48%|████▊     | 2390/5000 [00:05<00:06, 400.90it/s]
scatterplot with images:  49%|████▊     | 2431/5000 [00:05<00:06, 401.20it/s]
scatterplot with images:  49%|████▉     | 2472/5000 [00:06<00:06, 391.54it/s]
scatterplot with images:  50%|█████     | 2512/5000 [00:06<00:06, 393.81it/s]
scatterplot with images:  51%|█████     | 2553/5000 [00:06<00:06, 397.98it/s]
scatterplot with images:  52%|█████▏    | 2594/5000 [00:06<00:06, 399.46it/s]
scatterplot with images:  53%|█████▎    | 2635/5000 [00:06<00:05, 400.88it/s]
scatterplot with images:  54%|█████▎    | 2676/5000 [00:06<00:05, 402.46it/s]
scatterplot with images:  54%|█████▍    | 2717/5000 [00:06<00:05, 404.45it/s]
scatterplot with images:  55%|█████▌    | 2758/5000 [00:06<00:05, 401.25it/s]
scatterplot with images:  56%|█████▌    | 2799/5000 [00:06<00:05, 401.22it/s]
scatterplot with images:  57%|█████▋    | 2840/5000 [00:06<00:05, 401.97it/s]
scatterplot with images:  58%|█████▊    | 2881/5000 [00:07<00:05, 395.37it/s]
scatterplot with images:  58%|█████▊    | 2922/5000 [00:07<00:05, 397.59it/s]
scatterplot with images:  59%|█████▉    | 2963/5000 [00:07<00:05, 400.30it/s]
scatterplot with images:  60%|██████    | 3004/5000 [00:07<00:04, 400.32it/s]
scatterplot with images:  61%|██████    | 3045/5000 [00:07<00:04, 401.61it/s]
scatterplot with images:  62%|██████▏   | 3086/5000 [00:07<00:04, 400.55it/s]
scatterplot with images:  63%|██████▎   | 3127/5000 [00:07<00:04, 402.06it/s]
scatterplot with images:  63%|██████▎   | 3168/5000 [00:07<00:04, 402.49it/s]
scatterplot with images:  64%|██████▍   | 3209/5000 [00:08<00:11, 151.80it/s]
scatterplot with images:  65%|██████▍   | 3246/5000 [00:08<00:09, 184.42it/s]
scatterplot with images:  66%|██████▌   | 3284/5000 [00:08<00:07, 217.72it/s]
scatterplot with images:  66%|██████▋   | 3323/5000 [00:08<00:06, 250.16it/s]
scatterplot with images:  67%|██████▋   | 3362/5000 [00:08<00:05, 279.23it/s]
scatterplot with images:  68%|██████▊   | 3401/5000 [00:08<00:05, 304.24it/s]
scatterplot with images:  69%|██████▉   | 3440/5000 [00:09<00:04, 323.98it/s]
scatterplot with images:  70%|██████▉   | 3479/5000 [00:09<00:04, 339.90it/s]
scatterplot with images:  70%|███████   | 3518/5000 [00:09<00:04, 353.02it/s]
scatterplot with images:  71%|███████   | 3556/5000 [00:09<00:04, 358.01it/s]
scatterplot with images:  72%|███████▏  | 3596/5000 [00:09<00:03, 367.22it/s]
scatterplot with images:  73%|███████▎  | 3636/5000 [00:09<00:03, 375.56it/s]
scatterplot with images:  74%|███████▎  | 3677/5000 [00:09<00:03, 382.90it/s]
scatterplot with images:  74%|███████▍  | 3718/5000 [00:09<00:03, 388.29it/s]
scatterplot with images:  75%|███████▌  | 3758/5000 [00:09<00:03, 388.78it/s]
scatterplot with images:  76%|███████▌  | 3798/5000 [00:09<00:03, 391.85it/s]
scatterplot with images:  77%|███████▋  | 3838/5000 [00:10<00:02, 393.56it/s]
scatterplot with images:  78%|███████▊  | 3879/5000 [00:10<00:02, 395.59it/s]
scatterplot with images:  78%|███████▊  | 3920/5000 [00:10<00:02, 397.10it/s]
scatterplot with images:  79%|███████▉  | 3960/5000 [00:10<00:02, 391.67it/s]
scatterplot with images:  80%|████████  | 4001/5000 [00:10<00:02, 394.63it/s]
scatterplot with images:  81%|████████  | 4042/5000 [00:10<00:02, 397.78it/s]
scatterplot with images:  82%|████████▏ | 4083/5000 [00:10<00:02, 400.14it/s]
scatterplot with images:  82%|████████▏ | 4124/5000 [00:10<00:02, 400.56it/s]
scatterplot with images:  83%|████████▎ | 4165/5000 [00:10<00:02, 401.14it/s]
scatterplot with images:  84%|████████▍ | 4206/5000 [00:10<00:01, 400.32it/s]
scatterplot with images:  85%|████████▍ | 4247/5000 [00:11<00:01, 400.43it/s]
scatterplot with images:  86%|████████▌ | 4288/5000 [00:11<00:01, 400.72it/s]
scatterplot with images:  87%|████████▋ | 4329/5000 [00:11<00:01, 402.98it/s]
scatterplot with images:  87%|████████▋ | 4370/5000 [00:11<00:01, 397.16it/s]
scatterplot with images:  88%|████████▊ | 4411/5000 [00:11<00:01, 400.54it/s]
scatterplot with images:  89%|████████▉ | 4452/5000 [00:11<00:01, 401.60it/s]
scatterplot with images:  90%|████████▉ | 4493/5000 [00:11<00:01, 403.74it/s]
scatterplot with images:  91%|█████████ | 4534/5000 [00:11<00:01, 404.05it/s]
scatterplot with images:  92%|█████████▏| 4576/5000 [00:11<00:01, 406.21it/s]
scatterplot with images:  92%|█████████▏| 4617/5000 [00:11<00:00, 406.10it/s]
scatterplot with images:  93%|█████████▎| 4659/5000 [00:12<00:00, 408.01it/s]
scatterplot with images:  94%|█████████▍| 4700/5000 [00:12<00:00, 407.96it/s]
scatterplot with images:  95%|█████████▍| 4742/5000 [00:12<00:00, 408.85it/s]
scatterplot with images:  96%|█████████▌| 4783/5000 [00:12<00:00, 401.99it/s]
scatterplot with images:  96%|█████████▋| 4825/5000 [00:12<00:00, 405.66it/s]
scatterplot with images:  97%|█████████▋| 4867/5000 [00:12<00:00, 407.44it/s]
scatterplot with images:  98%|█████████▊| 4909/5000 [00:12<00:00, 409.90it/s]
scatterplot with images:  99%|█████████▉| 4951/5000 [00:12<00:00, 411.71it/s]
scatterplot with images: 100%|██████████| 5000/5000 [00:12<00:00, 386.80it/s]

channels:  75%|███████▌  | 3/4 [01:17<00:25, 25.92s/it]
scatterplot with images:   0%|          | 0/5000 [00:00<?, ?it/s]
scatterplot with images:   1%|          | 37/5000 [00:00<00:13, 363.05it/s]
scatterplot with images:   2%|▏         | 79/5000 [00:00<00:13, 378.37it/s]
scatterplot with images:   2%|▏         | 122/5000 [00:00<00:12, 390.57it/s]
scatterplot with images:   3%|▎         | 164/5000 [00:00<00:12, 397.04it/s]
scatterplot with images:   4%|▍         | 206/5000 [00:00<00:11, 403.18it/s]
scatterplot with images:   5%|▍         | 248/5000 [00:00<00:11, 406.81it/s]
scatterplot with images:   6%|▌         | 290/5000 [00:00<00:11, 409.58it/s]
scatterplot with images:   7%|▋         | 332/5000 [00:00<00:11, 411.14it/s]
scatterplot with images:   7%|▋         | 374/5000 [00:00<00:11, 413.36it/s]
scatterplot with images:   8%|▊         | 414/5000 [00:01<00:11, 406.13it/s]
scatterplot with images:   9%|▉         | 457/5000 [00:01<00:11, 410.79it/s]
scatterplot with images:  10%|▉         | 499/5000 [00:01<00:10, 413.07it/s]
scatterplot with images:  11%|█         | 542/5000 [00:01<00:10, 415.64it/s]
scatterplot with images:  12%|█▏        | 584/5000 [00:01<00:10, 416.02it/s]
scatterplot with images:  13%|█▎        | 626/5000 [00:01<00:10, 416.76it/s]
scatterplot with images:  13%|█▎        | 668/5000 [00:01<00:10, 417.54it/s]
scatterplot with images:  14%|█▍        | 710/5000 [00:01<00:10, 418.19it/s]
scatterplot with images:  15%|█▌        | 752/5000 [00:01<00:10, 417.10it/s]
scatterplot with images:  16%|█▌        | 795/5000 [00:01<00:10, 418.67it/s]
scatterplot with images:  17%|█▋        | 837/5000 [00:02<00:10, 411.40it/s]
scatterplot with images:  18%|█▊        | 879/5000 [00:02<00:09, 413.83it/s]
scatterplot with images:  18%|█▊        | 922/5000 [00:02<00:09, 415.71it/s]
scatterplot with images:  19%|█▉        | 965/5000 [00:02<00:09, 417.05it/s]
scatterplot with images:  20%|██        | 1007/5000 [00:02<00:09, 417.22it/s]
scatterplot with images:  21%|██        | 1049/5000 [00:02<00:09, 417.94it/s]
scatterplot with images:  22%|██▏       | 1091/5000 [00:02<00:09, 417.79it/s]
scatterplot with images:  23%|██▎       | 1133/5000 [00:02<00:09, 417.98it/s]
scatterplot with images:  24%|██▎       | 1176/5000 [00:02<00:09, 418.53it/s]
scatterplot with images:  24%|██▍       | 1219/5000 [00:02<00:09, 419.96it/s]
scatterplot with images:  25%|██▌       | 1261/5000 [00:03<00:09, 413.12it/s]
scatterplot with images:  26%|██▌       | 1303/5000 [00:03<00:08, 412.53it/s]
scatterplot with images:  27%|██▋       | 1345/5000 [00:03<00:08, 412.06it/s]
scatterplot with images:  28%|██▊       | 1387/5000 [00:03<00:08, 411.47it/s]
scatterplot with images:  29%|██▊       | 1429/5000 [00:03<00:08, 411.41it/s]
scatterplot with images:  29%|██▉       | 1471/5000 [00:04<00:22, 159.88it/s]
scatterplot with images:  30%|███       | 1508/5000 [00:04<00:18, 192.29it/s]
scatterplot with images:  31%|███       | 1545/5000 [00:04<00:15, 224.15it/s]
scatterplot with images:  32%|███▏      | 1583/5000 [00:04<00:13, 254.31it/s]
scatterplot with images:  32%|███▏      | 1620/5000 [00:04<00:12, 280.48it/s]
scatterplot with images:  33%|███▎      | 1659/5000 [00:04<00:10, 304.98it/s]
scatterplot with images:  34%|███▍      | 1697/5000 [00:04<00:10, 323.85it/s]
scatterplot with images:  35%|███▍      | 1736/5000 [00:04<00:09, 339.91it/s]
scatterplot with images:  35%|███▌      | 1774/5000 [00:04<00:09, 350.26it/s]
scatterplot with images:  36%|███▋      | 1813/5000 [00:05<00:08, 359.36it/s]
scatterplot with images:  37%|███▋      | 1851/5000 [00:05<00:08, 360.35it/s]
scatterplot with images:  38%|███▊      | 1890/5000 [00:05<00:08, 368.33it/s]
scatterplot with images:  39%|███▊      | 1930/5000 [00:05<00:08, 375.08it/s]
scatterplot with images:  39%|███▉      | 1970/5000 [00:05<00:07, 381.34it/s]
scatterplot with images:  40%|████      | 2010/5000 [00:05<00:07, 386.70it/s]
scatterplot with images:  41%|████      | 2050/5000 [00:05<00:07, 389.86it/s]
scatterplot with images:  42%|████▏     | 2090/5000 [00:05<00:07, 392.71it/s]
scatterplot with images:  43%|████▎     | 2130/5000 [00:05<00:07, 392.77it/s]
scatterplot with images:  43%|████▎     | 2170/5000 [00:05<00:07, 392.53it/s]
scatterplot with images:  44%|████▍     | 2210/5000 [00:06<00:07, 386.79it/s]
scatterplot with images:  45%|████▌     | 2250/5000 [00:06<00:07, 390.03it/s]
scatterplot with images:  46%|████▌     | 2290/5000 [00:06<00:06, 391.61it/s]
scatterplot with images:  47%|████▋     | 2330/5000 [00:06<00:06, 393.73it/s]
scatterplot with images:  47%|████▋     | 2370/5000 [00:06<00:06, 394.33it/s]
scatterplot with images:  48%|████▊     | 2410/5000 [00:06<00:06, 394.29it/s]
scatterplot with images:  49%|████▉     | 2450/5000 [00:06<00:06, 393.96it/s]
scatterplot with images:  50%|████▉     | 2490/5000 [00:06<00:06, 394.57it/s]
scatterplot with images:  51%|█████     | 2531/5000 [00:06<00:06, 396.92it/s]
scatterplot with images:  51%|█████▏    | 2571/5000 [00:06<00:06, 397.23it/s]
scatterplot with images:  52%|█████▏    | 2611/5000 [00:07<00:06, 392.27it/s]
scatterplot with images:  53%|█████▎    | 2652/5000 [00:07<00:05, 394.89it/s]
scatterplot with images:  54%|█████▍    | 2693/5000 [00:07<00:05, 397.74it/s]
scatterplot with images:  55%|█████▍    | 2733/5000 [00:07<00:05, 398.37it/s]
scatterplot with images:  55%|█████▌    | 2774/5000 [00:07<00:05, 399.21it/s]
scatterplot with images:  56%|█████▋    | 2815/5000 [00:07<00:05, 399.98it/s]
scatterplot with images:  57%|█████▋    | 2856/5000 [00:07<00:05, 401.04it/s]
scatterplot with images:  58%|█████▊    | 2897/5000 [00:07<00:05, 401.62it/s]
scatterplot with images:  59%|█████▉    | 2938/5000 [00:07<00:05, 404.03it/s]
scatterplot with images:  60%|█████▉    | 2979/5000 [00:07<00:04, 405.20it/s]
scatterplot with images:  60%|██████    | 3020/5000 [00:08<00:04, 398.74it/s]
scatterplot with images:  61%|██████    | 3061/5000 [00:08<00:04, 400.16it/s]
scatterplot with images:  62%|██████▏   | 3102/5000 [00:08<00:04, 403.01it/s]
scatterplot with images:  63%|██████▎   | 3143/5000 [00:08<00:04, 403.62it/s]
scatterplot with images:  64%|██████▎   | 3184/5000 [00:08<00:04, 405.23it/s]
scatterplot with images:  65%|██████▍   | 3226/5000 [00:08<00:04, 407.05it/s]
scatterplot with images:  65%|██████▌   | 3267/5000 [00:08<00:04, 407.64it/s]
scatterplot with images:  66%|██████▌   | 3309/5000 [00:08<00:04, 409.18it/s]
scatterplot with images:  67%|██████▋   | 3351/5000 [00:08<00:04, 410.20it/s]
scatterplot with images:  68%|██████▊   | 3393/5000 [00:08<00:03, 411.76it/s]
scatterplot with images:  69%|██████▊   | 3435/5000 [00:09<00:03, 404.60it/s]
scatterplot with images:  70%|██████▉   | 3477/5000 [00:09<00:03, 408.11it/s]
scatterplot with images:  70%|███████   | 3518/5000 [00:09<00:06, 233.01it/s]
scatterplot with images:  71%|███████   | 3560/5000 [00:09<00:05, 268.48it/s]
scatterplot with images:  72%|███████▏  | 3602/5000 [00:09<00:04, 300.31it/s]
scatterplot with images:  73%|███████▎  | 3644/5000 [00:09<00:04, 328.26it/s]
scatterplot with images:  74%|███████▎  | 3686/5000 [00:09<00:03, 350.65it/s]
scatterplot with images:  75%|███████▍  | 3726/5000 [00:10<00:03, 362.78it/s]
scatterplot with images:  75%|███████▌  | 3768/5000 [00:10<00:03, 377.91it/s]
scatterplot with images:  76%|███████▌  | 3811/5000 [00:10<00:03, 390.46it/s]
scatterplot with images:  77%|███████▋  | 3853/5000 [00:10<00:02, 398.04it/s]
scatterplot with images:  78%|███████▊  | 3895/5000 [00:10<00:02, 404.23it/s]
scatterplot with images:  79%|███████▊  | 3937/5000 [00:10<00:02, 407.78it/s]
scatterplot with images:  80%|███████▉  | 3979/5000 [00:10<00:02, 411.02it/s]
scatterplot with images:  80%|████████  | 4021/5000 [00:10<00:02, 413.45it/s]
scatterplot with images:  81%|████████▏ | 4063/5000 [00:10<00:02, 414.79it/s]
scatterplot with images:  82%|████████▏ | 4105/5000 [00:10<00:02, 415.54it/s]
scatterplot with images:  83%|████████▎ | 4147/5000 [00:11<00:02, 410.35it/s]
scatterplot with images:  84%|████████▍ | 4189/5000 [00:11<00:01, 412.43it/s]
scatterplot with images:  85%|████████▍ | 4231/5000 [00:11<00:01, 413.34it/s]
scatterplot with images:  85%|████████▌ | 4273/5000 [00:11<00:01, 414.62it/s]
scatterplot with images:  86%|████████▋ | 4316/5000 [00:11<00:01, 416.70it/s]
scatterplot with images:  87%|████████▋ | 4359/5000 [00:11<00:01, 417.75it/s]
scatterplot with images:  88%|████████▊ | 4402/5000 [00:11<00:01, 418.79it/s]
scatterplot with images:  89%|████████▉ | 4444/5000 [00:11<00:01, 418.87it/s]
scatterplot with images:  90%|████████▉ | 4487/5000 [00:11<00:01, 420.56it/s]
scatterplot with images:  91%|█████████ | 4530/5000 [00:11<00:01, 419.82it/s]
scatterplot with images:  91%|█████████▏| 4572/5000 [00:12<00:01, 411.65it/s]
scatterplot with images:  92%|█████████▏| 4614/5000 [00:12<00:00, 412.87it/s]
scatterplot with images:  93%|█████████▎| 4657/5000 [00:12<00:00, 415.18it/s]
scatterplot with images:  94%|█████████▍| 4699/5000 [00:12<00:00, 415.39it/s]
scatterplot with images:  95%|█████████▍| 4742/5000 [00:12<00:00, 416.80it/s]
scatterplot with images:  96%|█████████▌| 4784/5000 [00:12<00:00, 416.88it/s]
scatterplot with images:  97%|█████████▋| 4827/5000 [00:12<00:00, 418.22it/s]
scatterplot with images:  97%|█████████▋| 4869/5000 [00:12<00:00, 418.24it/s]
scatterplot with images:  98%|█████████▊| 4912/5000 [00:12<00:00, 419.34it/s]
scatterplot with images:  99%|█████████▉| 4954/5000 [00:12<00:00, 419.19it/s]
scatterplot with images: 100%|██████████| 5000/5000 [00:13<00:00, 382.03it/s]

channels: 100%|██████████| 4/4 [01:43<00:00, 25.93s/it]
In [12]:
# finding similar cells
all_expressions = []
for expression, _ in tqdm(cells_ds):
    all_expressions.append(expression.view(1, -1))
expressions = torch.cat(all_expressions, dim=0).numpy()

100%|██████████| 219095/219095 [00:02<00:00, 105489.47it/s]
In [13]:
aa = ad.AnnData(expressions)
sc.tl.pca(aa)
sc.pl.pca(aa)
In [14]:
bb = aa[random_indices]
In [15]:
print("computing umap... ", end="")
sc.pp.neighbors(bb)
sc.tl.umap(bb)
sc.tl.louvain(bb)
print("done")

computing umap... done
In [16]:
plt.figure()
l = bb.obs["louvain"].tolist()
colors = list(map(int, l))
plt.scatter(
    b.obsm["X_umap"][:, 0],
    b.obsm["X_umap"][:, 1],
    s=1,
    c=colors,
    cmap=matplotlib.cm.tab20,
)
plt.show()
In [17]:
plt.figure()
l = bb.obs["louvain"].tolist()
colors = list(map(int, l))
plt.scatter(
    bb.obsm["X_umap"][:, 0],
    bb.obsm["X_umap"][:, 1],
    s=1,
    c=colors,
    cmap=matplotlib.cm.tab20,
)
plt.show()
In [18]:
from sklearn.neighbors import NearestNeighbors
import numpy as np
X = expressions[random_indices[:p]]
nbrs = NearestNeighbors(n_neighbors=20, algorithm='ball_tree').fit(X)
distances, indices = nbrs.kneighbors(X)
indices
Out[18]:
array([[   0, 4427, 2544, ..., 1869, 2565, 4318],
       [   1, 2223, 1857, ..., 2218, 3372, 2866],
       [   2, 2108, 3919, ..., 4616, 3693, 4265],
       ...,
       [4997, 3652, 3616, ...,  351, 3428,   59],
       [4998, 4154,  872, ..., 4223, 1760, 2990],
       [4999,   97,  771, ...,  766, 2697,  994]])
In [19]:
some_cells = np.random.choice(np.arange(p), 5)
rows = len(some_cells)
cols = len(indices[42])
axes = plt.subplots(rows, cols, figsize=(30, 20))[1].flatten()
k = 0
for cell in tqdm(some_cells):
    for i in indices[cell]:
        ax = axes[k]
        e = expressions[i]
        ax.bar(np.arange(len(e)), e)
        ax.set(title=f'cell {i}, nn of {cell}')
        k += 1
plt.tight_layout()
plt.show()

100%|██████████| 5/5 [00:01<00:00,  2.57it/s]
In [20]:
# plot the nearest neighbors on the umaps, the points look extra sparse because the nearest neighbors are computed on
# a subset of points
for bbb in [b, bb]:
    axes = plt.subplots(1, rows, figsize=(15, 5))[1].flatten()
    k = 0
    for cell in tqdm(some_cells):
        nn = indices[cell]
        ax = axes[k]
        k += 1

        ax.scatter(
            bbb.obsm["X_umap"][:, 0],
            bbb.obsm["X_umap"][:, 1],
            s=1,
            color='k'
        )
        ax.scatter(
            bbb.obsm["X_umap"][nn, 0],
            bbb.obsm["X_umap"][nn, 1],
            s=1,
            color='r'
        )

    plt.tight_layout()
    plt.show()

100%|██████████| 5/5 [00:00<00:00, 334.63it/s]

100%|██████████| 5/5 [00:00<00:00, 350.02it/s]
In [ ]: